from airflow.hooks.spark_submit import SparkSubmitHook

hook = SparkSubmitHook(
    java_class='org.apache.spark.examples.SparkPi',
    executor_cores=1,
    executor_memory='1G',
    driver_memory='1G',
    proxy_user='test',
    name='test',
    num_executors=2,
    application_args=['1000'],
    spark_binary='/usr/hdp/3.1.5.0-152/spark3/bin/spark-submit',
    master='yarn',
    deploy_mode='cluster',
    queue='spark3',
    operator=None
)

hook.submit('spark-examples_2.11-2.3.2.3.1.5.0-152.jar')
